{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Deep Convolutional Generative Adversarial Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Software Installation\\Anaconda3\\lib\\site-packages\\h5py\\__init__.py:34: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def montage(images):\n",
    "    if isinstance(images, list):\n",
    "        images = np.array(images)\n",
    "    img_w = images.shape[1]\n",
    "    img_h = images.shape[2]\n",
    "    n_plots = int(np.ceil(np.sqrt(images.shape[0])))\n",
    "    m = np.ones((images.shape[1] * n_plots + n_plots + 1, images[2] * n_plots + n_plots + 1)) * 0.5\n",
    "    for i in range(n_plots):\n",
    "        for j in range(n_plots):\n",
    "            this_filter = i * n_plots + j\n",
    "            if this_filter < images.shape[0]:\n",
    "                this_img = images[this_filter]\n",
    "                m[1 + i + i * img_h : 1 + i + (i+1) * img_h,\n",
    "                 1 + j + j * img_w : 1 + j + (j+1) * img_w] = this_img\n",
    "    return m"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "setting up the basics\n",
    "构建数据集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-3-6774253fb94e>:2: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
      "WARNING:tensorflow:From C:\\Software Installation\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please write your own downloading logic.\n",
      "WARNING:tensorflow:From C:\\Software Installation\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting MNIST_data\\train-images-idx3-ubyte.gz\n",
      "WARNING:tensorflow:From C:\\Software Installation\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting MNIST_data\\train-labels-idx1-ubyte.gz\n",
      "Extracting MNIST_data\\t10k-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data\\t10k-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From C:\\Software Installation\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:290: DataSet.__init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets('MNIST_data')\n",
    "tf.reset_default_graph()\n",
    "batch_size = 64\n",
    "n_noise = 64\n",
    "\n",
    "X_in = tf.placeholder(dtype=tf.float32, shape=[None, 28, 28], name='X')\n",
    "noise = tf.placeholder(dtype=tf.float32, shape=[None, n_noise])\n",
    "\n",
    "keep_prob = tf.placeholder(dtype=tf.float32, name='keep_prob')\n",
    "is_training = tf.placeholder(dtype=tf.bool, name='is_training')\n",
    "\n",
    "def lrelu(x):\n",
    "    return tf.maximum(x, tf.multiply(x, 0.2))\n",
    "\n",
    "def binary_cross_entropy(x, z):\n",
    "    eps = 1e-12\n",
    "    return (-(x * tf.log(z + eps) + (1. - x) * tf.log(1. - z + eps)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "定义判别器discriminator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def discriminator(img_in, reuse=None, keep_prob=keep_prob):\n",
    "    activation = lrelu\n",
    "    with tf.variable_scope(\"discriminator\", reuse=reuse):\n",
    "        x = tf.reshape(img_in, shape=[-1, 28, 28, 1])\n",
    "        x = tf.layers.conv2d(x, kernel_size=5, filters=64, strides=2, padding='same', activation=activation)\n",
    "        x = tf.layers.dropout(x, keep_prob)\n",
    "        x = tf.layers.conv2d(x, kernel_size=5, filters=64, strides=1, padding='same', activation=activation)\n",
    "        x = tf.layers.dropout(x, keep_prob)\n",
    "        x = tf.layers.conv2d(x, kernel_size=5, filters=64, strides=1, padding='same', activation=activation)\n",
    "        x = tf.layers.dropout(x, keep_prob)\n",
    "        x = tf.contrib.layers.flatten(x)\n",
    "        x = tf.layers.dense(x, units=128, activation=activation)\n",
    "        x = tf.layers.dense(x, units=1, activation=tf.nn.sigmoid)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "定义生成器generator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def generator(z, keep_prob=keep_prob, is_training=is_training):\n",
    "    activation = lrelu\n",
    "    momentum = 0.99\n",
    "    with tf.variable_scope(\"generator\", reuse=None):\n",
    "        x = z\n",
    "        d1 = 4\n",
    "        d2 = 1\n",
    "        x = tf.layers.dense(x, units= d1 * d1 * d2, activation=activation)\n",
    "        x = tf.layers.dropout(x, keep_prob)\n",
    "        x = tf.contrib.layers.batch_norm(x, is_training=is_training, decay=momentum)\n",
    "        x = tf.reshape(x, shape=[-1, d1, d1, d2])\n",
    "        x = tf.image.resize_images(x, size=[7, 7])\n",
    "        x = tf.layers.conv2d_transpose(x, kernel_size=5, filters=64, strides=2, padding='same', activation=activation)\n",
    "        x = tf.layers.dropout(x, keep_prob)\n",
    "        x = tf.contrib.layers.batch_norm(x, is_training=is_training, decay=momentum)\n",
    "        x = tf.layers.conv2d_transpose(x, kernel_size=5, filters=64, strides=2, padding='same', activation=activation)\n",
    "        x = tf.layers.dropout(x, keep_prob)\n",
    "        x = tf.contrib.layers.batch_norm(x, is_training=is_training, decay=momentum)\n",
    "        x = tf.layers.conv2d_transpose(x, kernel_size=5, filters=64, strides=1, padding='same', activation=activation)\n",
    "        x = tf.layers.dropout(x, keep_prob)\n",
    "        x = tf.contrib.layers.batch_norm(x, is_training=is_training, decay=momentum)\n",
    "        x = tf.layers.conv2d_transpose(x, kernel_size=5, filters=64, strides=1, padding='same', activation=activation)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "定义损失函数和优化器"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "g = generator(noise, keep_prob, is_training)\n",
    "d_real = discriminator(X_in)\n",
    "d_fake = discriminator(g, reuse=True)\n",
    "\n",
    "vars_g = [var for var in tf.trainable_variables() if var.name.startswith('generator')]\n",
    "vars_d = [var for var in tf.trainable_variables() if var.name.startswith('discriminator')]\n",
    "\n",
    "d_reg = tf.contrib.layers.apply_regularization(tf.contrib.layers.l2_regularizer(1e-6), vars_d)\n",
    "g_reg = tf.contrib.layers.apply_regularization(tf.contrib.layers.l2_regularizer(1e-6), vars_g)\n",
    "\n",
    "loss_d_real = binary_cross_entropy(tf.ones_like(d_real), d_real)\n",
    "loss_d_fake = binary_cross_entropy(tf.zeros_like(d_fake), d_fake)\n",
    "loss_g = tf.reduce_mean(binary_cross_entropy(tf.ones_like(d_fake), d_fake))\n",
    "loss_d = tf.reduce_mean(0.5 * (loss_d_real + loss_d_fake))\n",
    "\n",
    "update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)\n",
    "with tf.control_dependencies(update_ops):\n",
    "    optimizer_d = tf.train.RMSPropOptimizer(learning_rate=0.00015).minimize(loss_d + d_reg, var_list=vars_d)\n",
    "    optimizer_g = tf.train.RMSPropOptimizer(learning_rate=0.00015).minimize(loss_g + g_reg, var_list=vars_g)\n",
    "    \n",
    "sess = tf.Session()\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "训练网络"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "unexpected EOF while parsing (<ipython-input-7-c0ae3ea0b857>, line 23)",
     "output_type": "error",
     "traceback": [
      "\u001b[1;36m  File \u001b[1;32m\"<ipython-input-7-c0ae3ea0b857>\"\u001b[1;36m, line \u001b[1;32m23\u001b[0m\n\u001b[1;33m    if not i % 50:\u001b[0m\n\u001b[1;37m                  ^\u001b[0m\n\u001b[1;31mSyntaxError\u001b[0m\u001b[1;31m:\u001b[0m unexpected EOF while parsing\n"
     ]
    }
   ],
   "source": [
    "for i in range(60000):\n",
    "    train_d = True\n",
    "    train_g = True\n",
    "    keep_prob_train = 0.6\n",
    "    \n",
    "    n = np.random.uniform(0.0, 1.0 [batch_size, n_noise]).astype(np.float32)\n",
    "    batch = [np.reshape(b, [28, 28]) for b in mnist.train.next_batch(batch_size=batch_size)[0]]\n",
    "    d_real_ls, d_fake_ls, g_ls, d_ls = sess.run([loss_d_real, loss_d_fake, loss_g, loss_d], feed_dict={X_in:batch, noise:n, keep_prob:keep_prob_train, is_training:True})\n",
    "    d_real_ls = np.mean(d_real_ls)\n",
    "    d_fake_ls = np.mean(d_fake_ls)\n",
    "    if g_ls * 1.5 < d_ls:\n",
    "        train_g = False\n",
    "        pass\n",
    "    if d_ls * 2 < g_ls:\n",
    "        train_d = False\n",
    "        pass\n",
    "    \n",
    "    if train_d:\n",
    "        sess.run(optimizer_d, feed_dict={noise:n, X_in:batch, keep_prob:keep_prob_train, is_training:True})\n",
    "    if train_g:\n",
    "        sess.run(optimizer_g, feed_dict={noise:n, keep_prob:keep_prob_train, is_training:True})\n",
    "    \n",
    "    if not i % 50:\n",
    "        print(i, d_ls, g_ls, d_real_ls, d_fake_ls)\n",
    "        if not train_g:\n",
    "            print('not training generator')\n",
    "        if not train_d:\n",
    "            print('not training descriminator')\n",
    "        gen_img = sess.run(f, feed_dict = {noise:n, keep_pron:1.0, is_trainig:False})\n",
    "        imgs = [img[:,:,0] for img in gen_img]\n",
    "        m = montage(imgs)\n",
    "        gen_img = m\n",
    "        plt.axis('off')\n",
    "        plt.imshow(gen_img, cmap='gray')\n",
    "        ply.show()      "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
